In [1]:
TRIAL_ID = '32a_n50'
USE_TBX = False
CHECKPOINT = None
COEFF_REC = 1#e-3
COEFF_KLD = 1e5 # beta
In [2]:
import torch
import torch.utils.data
from torch import nn, optim
from torch.nn import functional as F
from torchvision import transforms
from torchvision.utils import make_grid
from torch.utils.data import Dataset,DataLoader
import pathlib
import matplotlib.pyplot as plt
import numpy as np
import os
from utils import plot_3d_mesh, VertDataset, ResizeTo
import utils as ut
%load_ext autoreload
%autoreload 2
%matplotlib inline
In [3]:
#Define paramaters
VOL_SIZE = 32
EPOCHS = 50
LEARNING_RATE = 1e-3
LOG_INTERVAL = 3
N = None # None for full dataset
BATCH_SIZE = 64
Z_DIMS = 64
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
train_path = pathlib.Path("./for_VAE/npz")
results_dir = pathlib.Path("./results/%s" % TRIAL_ID)
results_dir.mkdir(parents=True, exist_ok=True)
In [4]:
if USE_TBX:
from tensorboardX import SummaryWriter
#SummaryWriter encapsulates everything
log_dir = pathlib.Path('./logs/%s' % TRIAL_ID)
writer = SummaryWriter(log_dir)
In [5]:
dataset = VertDataset(train_path,n=N, transform=transforms.Compose([ResizeTo(VOL_SIZE),
transforms.ToTensor()]))
print('Number of samples: ',len(dataset))
train_loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
print('Number of batches: ',len(train_loader))
Number of samples: 1427 Number of batches: 23
In [6]:
# x = dataset[0]
# print(x.size())
# print(x.unique())
# plot_3d_mesh(x)
# del x
In [7]:
from vae32 import Vae32 as VAE
model = VAE(debug=False).to(device)
weights_init = ut.init_weights(init_type='normal')
model.apply(weights_init)
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
In [8]:
def loss_function(recon_x, x, mu, logvar):
batch_size = recon_x.size(0)
#measure of how well reconstruction matches input
rec_loss = F.binary_cross_entropy(recon_x, x, reduction='sum')
#measure of distance btw learned distribution and Normal Gaussian
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
#Normalize over batches
rec_loss = rec_loss / batch_size
KLD = KLD / batch_size
rec_loss *= COEFF_REC
KLD *= COEFF_KLD
loss = rec_loss + KLD
return loss, rec_loss, KLD
In [9]:
# Training setup
loss_history = list()
rec_loss_history = list()
kl_loss_history = list()
iters=0
N_batches_train = len(train_loader)
start_epoch = 0
if CHECKPOINT is not None:
checkpoint_path = '%s/%s' % (results_dir, CHECKPOINT)
model, optimizer, start_epoch, loss_history, rec_loss_history, kl_loss_history = ut.load_checkpoint(model, optimizer,filename = checkpoint_path)
iters = start_epoch * N_batches_train
model = model.to(device)
for state in optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
state[k] = v.to(device)
In [10]:
import time
end_epoch = start_epoch + EPOCHS
epoch = start_epoch
for epoch in range(start_epoch, end_epoch):
start = time.time()
################## TRAIN ########################
model.train()
train_loss = 0
for batch_idx, data in enumerate(train_loader):
data = data.to(device)
optimizer.zero_grad()
recon_batch, mu, logvar = model(data)
loss, rec_loss, kl_loss = loss_function(recon_batch, data, mu, logvar)
if USE_TBX:
writer.add_scalar('Loss/Rec',rec_loss.detach().item(),iters)
writer.add_scalar('Loss/KL',kl_loss.detach().item(),iters)
writer.add_scalar('Loss/Total_Loss',loss.detach().item(),iters)
loss.backward()
loss_history.append( loss.item() )
rec_loss_history.append( rec_loss.item() )
kl_loss_history.append( kl_loss.item() )
train_loss += loss.item()
optimizer.step()
if batch_idx % LOG_INTERVAL == 0:
print('Train Epoch: {:02d} [{:04d}/{:04d} ({:.0f}%)]\tRec: {:.3f}, KLD: {:.6f}, Loss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader),
rec_loss.item(), kl_loss.item(), loss.item() ))
iters+=1
#Finished one epoch
print('====> Epoch: {:02d} Average loss: {:.5f}'.format(
epoch, train_loss / len(train_loader.dataset)))
end = time.time()
print('Training time for epoch: {:.2f}s'.format(end-start))
print('======================================================')
Train Epoch: 00 [0000/1427 (0%)] Rec: 22712.930, KLD: 0.094250, Loss: 22713.023438 Train Epoch: 00 [0192/1427 (13%)] Rec: 22564.582, KLD: 0.905385, Loss: 22565.488281 Train Epoch: 00 [0384/1427 (26%)] Rec: 22138.527, KLD: 0.892859, Loss: 22139.419922 Train Epoch: 00 [0576/1427 (39%)] Rec: 20730.227, KLD: 0.825524, Loss: 20731.052734 Train Epoch: 00 [0768/1427 (52%)] Rec: 19403.469, KLD: 0.429153, Loss: 19403.898438 Train Epoch: 00 [0960/1427 (65%)] Rec: 19174.643, KLD: 0.771880, Loss: 19175.414062 Train Epoch: 00 [1152/1427 (78%)] Rec: 18021.965, KLD: 0.834465, Loss: 18022.798828 Train Epoch: 00 [1344/1427 (91%)] Rec: 17730.959, KLD: 0.244379, Loss: 17731.203125 ====> Epoch: 00 Average loss: 326.82477 Training time for epoch: 35.85s ====================================================== Train Epoch: 01 [0000/1427 (0%)] Rec: 17249.211, KLD: 0.482798, Loss: 17249.693359 Train Epoch: 01 [0192/1427 (13%)] Rec: 16569.338, KLD: 0.223517, Loss: 16569.560547 Train Epoch: 01 [0384/1427 (26%)] Rec: 15540.149, KLD: 0.429153, Loss: 15540.578125 Train Epoch: 01 [0576/1427 (39%)] Rec: 13940.895, KLD: 0.303984, Loss: 13941.198242 Train Epoch: 01 [0768/1427 (52%)] Rec: 13267.347, KLD: 0.169873, Loss: 13267.516602 Train Epoch: 01 [0960/1427 (65%)] Rec: 12970.047, KLD: 0.199676, Loss: 12970.246094 Train Epoch: 01 [1152/1427 (78%)] Rec: 11841.427, KLD: 0.077486, Loss: 11841.503906 Train Epoch: 01 [1344/1427 (91%)] Rec: 11541.532, KLD: 0.047684, Loss: 11541.580078 ====> Epoch: 01 Average loss: 226.22622 Training time for epoch: 27.58s ====================================================== Train Epoch: 02 [0000/1427 (0%)] Rec: 11321.208, KLD: 0.080466, Loss: 11321.288086 Train Epoch: 02 [0192/1427 (13%)] Rec: 11229.955, KLD: 0.032783, Loss: 11229.988281 Train Epoch: 02 [0384/1427 (26%)] Rec: 10918.050, KLD: 0.038743, Loss: 10918.088867 Train Epoch: 02 [0576/1427 (39%)] Rec: 10082.676, KLD: 0.044703, Loss: 10082.720703 Train Epoch: 02 [0768/1427 (52%)] Rec: 9911.516, KLD: 0.032783, Loss: 9911.548828 Train Epoch: 02 [0960/1427 (65%)] Rec: 9504.358, KLD: 0.035763, Loss: 9504.394531 Train Epoch: 02 [1152/1427 (78%)] Rec: 8562.382, KLD: 0.008941, Loss: 8562.390625 Train Epoch: 02 [1344/1427 (91%)] Rec: 8821.342, KLD: 0.062585, Loss: 8821.404297 ====> Epoch: 02 Average loss: 161.64693 Training time for epoch: 27.69s ====================================================== Train Epoch: 03 [0000/1427 (0%)] Rec: 8548.026, KLD: 0.032783, Loss: 8548.059570 Train Epoch: 03 [0192/1427 (13%)] Rec: 8343.333, KLD: -0.008941, Loss: 8343.324219 Train Epoch: 03 [0384/1427 (26%)] Rec: 7840.441, KLD: 0.017881, Loss: 7840.458984 Train Epoch: 03 [0576/1427 (39%)] Rec: 7396.571, KLD: -0.000000, Loss: 7396.571289 Train Epoch: 03 [0768/1427 (52%)] Rec: 7187.246, KLD: 0.014901, Loss: 7187.261230 Train Epoch: 03 [0960/1427 (65%)] Rec: 6879.298, KLD: 0.074506, Loss: 6879.372559 Train Epoch: 03 [1152/1427 (78%)] Rec: 6280.306, KLD: -0.032783, Loss: 6280.272949 Train Epoch: 03 [1344/1427 (91%)] Rec: 7454.211, KLD: -0.002980, Loss: 7454.208008 ====> Epoch: 03 Average loss: 120.12005 Training time for epoch: 27.28s ====================================================== Train Epoch: 04 [0000/1427 (0%)] Rec: 6479.132, KLD: 0.041723, Loss: 6479.173828 Train Epoch: 04 [0192/1427 (13%)] Rec: 6388.413, KLD: 0.008941, Loss: 6388.421875 Train Epoch: 04 [0384/1427 (26%)] Rec: 5963.763, KLD: 0.059605, Loss: 5963.822266 Train Epoch: 04 [0576/1427 (39%)] Rec: 6121.760, KLD: 0.050664, Loss: 6121.810547 Train Epoch: 04 [0768/1427 (52%)] Rec: 5953.533, KLD: 0.005960, Loss: 5953.539062 Train Epoch: 04 [0960/1427 (65%)] Rec: 5809.315, KLD: 0.005960, Loss: 5809.321289 Train Epoch: 04 [1152/1427 (78%)] Rec: 5430.664, KLD: 0.050664, Loss: 5430.714844 Train Epoch: 04 [1344/1427 (91%)] Rec: 6733.854, KLD: 0.023842, Loss: 6733.878418 ====> Epoch: 04 Average loss: 98.64484 Training time for epoch: 27.07s ====================================================== Train Epoch: 05 [0000/1427 (0%)] Rec: 5670.837, KLD: 0.083447, Loss: 5670.920898 Train Epoch: 05 [0192/1427 (13%)] Rec: 5704.775, KLD: -0.032783, Loss: 5704.742676 Train Epoch: 05 [0384/1427 (26%)] Rec: 5367.868, KLD: 0.110269, Loss: 5367.978027 Train Epoch: 05 [0576/1427 (39%)] Rec: 5523.396, KLD: 0.029802, Loss: 5523.425781 Train Epoch: 05 [0768/1427 (52%)] Rec: 5376.932, KLD: 0.035763, Loss: 5376.967773 Train Epoch: 05 [0960/1427 (65%)] Rec: 5323.840, KLD: 0.035763, Loss: 5323.875488 Train Epoch: 05 [1152/1427 (78%)] Rec: 5061.732, KLD: 0.017881, Loss: 5061.750488 Train Epoch: 05 [1344/1427 (91%)] Rec: 5964.813, KLD: 0.032783, Loss: 5964.846191 ====> Epoch: 05 Average loss: 89.01162 Training time for epoch: 27.08s ====================================================== Train Epoch: 06 [0000/1427 (0%)] Rec: 5312.695, KLD: 0.047684, Loss: 5312.743164 Train Epoch: 06 [0192/1427 (13%)] Rec: 5302.816, KLD: 0.014901, Loss: 5302.831543 Train Epoch: 06 [0384/1427 (26%)] Rec: 5044.586, KLD: 0.047684, Loss: 5044.634277 Train Epoch: 06 [0576/1427 (39%)] Rec: 5208.708, KLD: 0.041723, Loss: 5208.749023 Train Epoch: 06 [0768/1427 (52%)] Rec: 5134.695, KLD: 0.023842, Loss: 5134.718750 Train Epoch: 06 [0960/1427 (65%)] Rec: 5074.701, KLD: 0.032783, Loss: 5074.733887 Train Epoch: 06 [1152/1427 (78%)] Rec: 4764.527, KLD: -0.026822, Loss: 4764.500000 Train Epoch: 06 [1344/1427 (91%)] Rec: 5732.596, KLD: 0.053644, Loss: 5732.649414 ====> Epoch: 06 Average loss: 84.11738 Training time for epoch: 27.14s ====================================================== Train Epoch: 07 [0000/1427 (0%)] Rec: 5077.019, KLD: 0.023842, Loss: 5077.042969 Train Epoch: 07 [0192/1427 (13%)] Rec: 5159.650, KLD: 0.023842, Loss: 5159.674316 Train Epoch: 07 [0384/1427 (26%)] Rec: 4870.826, KLD: 0.014901, Loss: 4870.840820 Train Epoch: 07 [0576/1427 (39%)] Rec: 5013.411, KLD: 0.020862, Loss: 5013.431641 Train Epoch: 07 [0768/1427 (52%)] Rec: 5011.662, KLD: -0.008941, Loss: 5011.653320 Train Epoch: 07 [0960/1427 (65%)] Rec: 4965.344, KLD: -0.005960, Loss: 4965.338379 Train Epoch: 07 [1152/1427 (78%)] Rec: 4641.624, KLD: 0.014901, Loss: 4641.639160 Train Epoch: 07 [1344/1427 (91%)] Rec: 5570.546, KLD: 0.014901, Loss: 5570.561035 ====> Epoch: 07 Average loss: 81.62963 Training time for epoch: 27.04s ====================================================== Train Epoch: 08 [0000/1427 (0%)] Rec: 4939.741, KLD: 0.002980, Loss: 4939.744141 Train Epoch: 08 [0192/1427 (13%)] Rec: 5044.171, KLD: 0.017881, Loss: 5044.188965 Train Epoch: 08 [0384/1427 (26%)] Rec: 4771.714, KLD: 0.026822, Loss: 4771.740723 Train Epoch: 08 [0576/1427 (39%)] Rec: 4913.103, KLD: 0.005960, Loss: 4913.108398 Train Epoch: 08 [0768/1427 (52%)] Rec: 4903.656, KLD: 0.041723, Loss: 4903.697754 Train Epoch: 08 [0960/1427 (65%)] Rec: 4859.773, KLD: -0.000000, Loss: 4859.773438 Train Epoch: 08 [1152/1427 (78%)] Rec: 4584.717, KLD: -0.002980, Loss: 4584.713867 Train Epoch: 08 [1344/1427 (91%)] Rec: 5574.780, KLD: 0.029802, Loss: 5574.809570 ====> Epoch: 08 Average loss: 80.15231 Training time for epoch: 27.63s ====================================================== Train Epoch: 09 [0000/1427 (0%)] Rec: 4852.171, KLD: 0.005960, Loss: 4852.177246 Train Epoch: 09 [0192/1427 (13%)] Rec: 4972.879, KLD: 0.035763, Loss: 4972.914551 Train Epoch: 09 [0384/1427 (26%)] Rec: 4703.804, KLD: 0.020862, Loss: 4703.824707 Train Epoch: 09 [0576/1427 (39%)] Rec: 4867.156, KLD: 0.038743, Loss: 4867.194824 Train Epoch: 09 [0768/1427 (52%)] Rec: 4846.809, KLD: 0.023842, Loss: 4846.832520 Train Epoch: 09 [0960/1427 (65%)] Rec: 4784.922, KLD: 0.002980, Loss: 4784.925293 Train Epoch: 09 [1152/1427 (78%)] Rec: 4497.622, KLD: -0.000000, Loss: 4497.622070 Train Epoch: 09 [1344/1427 (91%)] Rec: 5500.845, KLD: -0.005960, Loss: 5500.839355 ====> Epoch: 09 Average loss: 78.94528 Training time for epoch: 30.39s ====================================================== Train Epoch: 10 [0000/1427 (0%)] Rec: 4775.886, KLD: 0.008941, Loss: 4775.895020 Train Epoch: 10 [0192/1427 (13%)] Rec: 4891.505, KLD: -0.002980, Loss: 4891.501953 Train Epoch: 10 [0384/1427 (26%)] Rec: 4651.389, KLD: -0.005960, Loss: 4651.382812 Train Epoch: 10 [0576/1427 (39%)] Rec: 4799.814, KLD: -0.000000, Loss: 4799.814453 Train Epoch: 10 [0768/1427 (52%)] Rec: 4774.164, KLD: 0.044703, Loss: 4774.208984 Train Epoch: 10 [0960/1427 (65%)] Rec: 4723.945, KLD: -0.000000, Loss: 4723.945312 Train Epoch: 10 [1152/1427 (78%)] Rec: 4453.648, KLD: 0.008941, Loss: 4453.657227 Train Epoch: 10 [1344/1427 (91%)] Rec: 5517.119, KLD: -0.000000, Loss: 5517.118652 ====> Epoch: 10 Average loss: 78.00311 Training time for epoch: 28.17s ====================================================== Train Epoch: 11 [0000/1427 (0%)] Rec: 4706.691, KLD: 0.029802, Loss: 4706.721191 Train Epoch: 11 [0192/1427 (13%)] Rec: 4827.961, KLD: 0.047684, Loss: 4828.008789 Train Epoch: 11 [0384/1427 (26%)] Rec: 4598.950, KLD: 0.002980, Loss: 4598.953125 Train Epoch: 11 [0576/1427 (39%)] Rec: 4758.981, KLD: 0.020862, Loss: 4759.002441 Train Epoch: 11 [0768/1427 (52%)] Rec: 4689.386, KLD: 0.047684, Loss: 4689.433594 Train Epoch: 11 [0960/1427 (65%)] Rec: 4657.137, KLD: 0.026822, Loss: 4657.163574 Train Epoch: 11 [1152/1427 (78%)] Rec: 4397.353, KLD: -0.005960, Loss: 4397.347168 Train Epoch: 11 [1344/1427 (91%)] Rec: 5467.467, KLD: 0.020862, Loss: 5467.488281 ====> Epoch: 11 Average loss: 77.06505 Training time for epoch: 27.50s ====================================================== Train Epoch: 12 [0000/1427 (0%)] Rec: 4680.294, KLD: 0.035763, Loss: 4680.330078 Train Epoch: 12 [0192/1427 (13%)] Rec: 4749.666, KLD: 0.011921, Loss: 4749.677246 Train Epoch: 12 [0384/1427 (26%)] Rec: 4560.227, KLD: 0.032783, Loss: 4560.259766 Train Epoch: 12 [0576/1427 (39%)] Rec: 4702.076, KLD: 0.011921, Loss: 4702.087891 Train Epoch: 12 [0768/1427 (52%)] Rec: 4653.309, KLD: -0.008941, Loss: 4653.300293 Train Epoch: 12 [0960/1427 (65%)] Rec: 4615.877, KLD: 0.056624, Loss: 4615.933594 Train Epoch: 12 [1152/1427 (78%)] Rec: 4350.156, KLD: 0.008941, Loss: 4350.165039 Train Epoch: 12 [1344/1427 (91%)] Rec: 5387.752, KLD: 0.059605, Loss: 5387.811523 ====> Epoch: 12 Average loss: 76.21603 Training time for epoch: 27.78s ====================================================== Train Epoch: 13 [0000/1427 (0%)] Rec: 4637.729, KLD: 0.053644, Loss: 4637.782227 Train Epoch: 13 [0192/1427 (13%)] Rec: 4690.195, KLD: 0.008941, Loss: 4690.204102 Train Epoch: 13 [0384/1427 (26%)] Rec: 4524.246, KLD: 0.032783, Loss: 4524.278809 Train Epoch: 13 [0576/1427 (39%)] Rec: 4629.872, KLD: 0.008941, Loss: 4629.880859 Train Epoch: 13 [0768/1427 (52%)] Rec: 4603.085, KLD: 0.014901, Loss: 4603.100098 Train Epoch: 13 [0960/1427 (65%)] Rec: 4547.033, KLD: 0.032783, Loss: 4547.065430 Train Epoch: 13 [1152/1427 (78%)] Rec: 4323.325, KLD: 0.005960, Loss: 4323.330566 Train Epoch: 13 [1344/1427 (91%)] Rec: 5280.694, KLD: 0.026822, Loss: 5280.721191 ====> Epoch: 13 Average loss: 75.37578 Training time for epoch: 27.85s ====================================================== Train Epoch: 14 [0000/1427 (0%)] Rec: 4583.904, KLD: 0.047684, Loss: 4583.952148 Train Epoch: 14 [0192/1427 (13%)] Rec: 4634.219, KLD: -0.000000, Loss: 4634.218750 Train Epoch: 14 [0384/1427 (26%)] Rec: 4506.173, KLD: 0.014901, Loss: 4506.188477 Train Epoch: 14 [0576/1427 (39%)] Rec: 4581.875, KLD: -0.002980, Loss: 4581.872070 Train Epoch: 14 [0768/1427 (52%)] Rec: 4557.595, KLD: -0.005960, Loss: 4557.589355 Train Epoch: 14 [0960/1427 (65%)] Rec: 4490.820, KLD: 0.008941, Loss: 4490.829102 Train Epoch: 14 [1152/1427 (78%)] Rec: 4279.041, KLD: 0.038743, Loss: 4279.079590 Train Epoch: 14 [1344/1427 (91%)] Rec: 5220.089, KLD: -0.023842, Loss: 5220.065430 ====> Epoch: 14 Average loss: 74.70104 Training time for epoch: 27.55s ====================================================== Train Epoch: 15 [0000/1427 (0%)] Rec: 4525.348, KLD: 0.053644, Loss: 4525.401855 Train Epoch: 15 [0192/1427 (13%)] Rec: 4588.120, KLD: 0.071526, Loss: 4588.190918 Train Epoch: 15 [0384/1427 (26%)] Rec: 4462.256, KLD: 0.017881, Loss: 4462.273926 Train Epoch: 15 [0576/1427 (39%)] Rec: 4552.113, KLD: 0.014901, Loss: 4552.127930 Train Epoch: 15 [0768/1427 (52%)] Rec: 4504.563, KLD: 0.020862, Loss: 4504.584473 Train Epoch: 15 [0960/1427 (65%)] Rec: 4466.461, KLD: -0.020862, Loss: 4466.439941 Train Epoch: 15 [1152/1427 (78%)] Rec: 4235.850, KLD: 0.038743, Loss: 4235.888184 Train Epoch: 15 [1344/1427 (91%)] Rec: 5255.308, KLD: 0.005960, Loss: 5255.313477 ====> Epoch: 15 Average loss: 74.12512 Training time for epoch: 27.74s ====================================================== Train Epoch: 16 [0000/1427 (0%)] Rec: 4483.449, KLD: 0.014901, Loss: 4483.464355 Train Epoch: 16 [0192/1427 (13%)] Rec: 4560.915, KLD: 0.008941, Loss: 4560.923828 Train Epoch: 16 [0384/1427 (26%)] Rec: 4432.513, KLD: 0.008941, Loss: 4432.521973 Train Epoch: 16 [0576/1427 (39%)] Rec: 4526.313, KLD: 0.020862, Loss: 4526.333984 Train Epoch: 16 [0768/1427 (52%)] Rec: 4483.000, KLD: -0.008941, Loss: 4482.991211 Train Epoch: 16 [0960/1427 (65%)] Rec: 4412.695, KLD: 0.029802, Loss: 4412.725098 Train Epoch: 16 [1152/1427 (78%)] Rec: 4199.330, KLD: 0.074506, Loss: 4199.404785 Train Epoch: 16 [1344/1427 (91%)] Rec: 5233.969, KLD: 0.026822, Loss: 5233.995605 ====> Epoch: 16 Average loss: 73.64673 Training time for epoch: 27.99s ====================================================== Train Epoch: 17 [0000/1427 (0%)] Rec: 4454.730, KLD: 0.083447, Loss: 4454.813965 Train Epoch: 17 [0192/1427 (13%)] Rec: 4534.180, KLD: 0.035763, Loss: 4534.215820 Train Epoch: 17 [0384/1427 (26%)] Rec: 4418.998, KLD: 0.029802, Loss: 4419.027344 Train Epoch: 17 [0576/1427 (39%)] Rec: 4520.135, KLD: 0.035763, Loss: 4520.170898 Train Epoch: 17 [0768/1427 (52%)] Rec: 4445.585, KLD: 0.029802, Loss: 4445.614746 Train Epoch: 17 [0960/1427 (65%)] Rec: 4384.638, KLD: 0.026822, Loss: 4384.665039 Train Epoch: 17 [1152/1427 (78%)] Rec: 4188.834, KLD: 0.035763, Loss: 4188.869629 Train Epoch: 17 [1344/1427 (91%)] Rec: 5213.080, KLD: -0.000000, Loss: 5213.080078 ====> Epoch: 17 Average loss: 73.37893 Training time for epoch: 28.25s ====================================================== Train Epoch: 18 [0000/1427 (0%)] Rec: 4422.874, KLD: 0.032783, Loss: 4422.906738 Train Epoch: 18 [0192/1427 (13%)] Rec: 4476.388, KLD: 0.059605, Loss: 4476.447754 Train Epoch: 18 [0384/1427 (26%)] Rec: 4416.237, KLD: 0.080466, Loss: 4416.317383 Train Epoch: 18 [0576/1427 (39%)] Rec: 4512.427, KLD: 0.023842, Loss: 4512.451172 Train Epoch: 18 [0768/1427 (52%)] Rec: 4425.440, KLD: 0.002980, Loss: 4425.443359 Train Epoch: 18 [0960/1427 (65%)] Rec: 4349.071, KLD: 0.020862, Loss: 4349.092285 Train Epoch: 18 [1152/1427 (78%)] Rec: 4170.085, KLD: 0.035763, Loss: 4170.120605 Train Epoch: 18 [1344/1427 (91%)] Rec: 5190.483, KLD: 0.005960, Loss: 5190.489258 ====> Epoch: 18 Average loss: 72.97825 Training time for epoch: 28.26s ====================================================== Train Epoch: 19 [0000/1427 (0%)] Rec: 4410.044, KLD: 0.029802, Loss: 4410.073730 Train Epoch: 19 [0192/1427 (13%)] Rec: 4441.207, KLD: 0.044703, Loss: 4441.251953 Train Epoch: 19 [0384/1427 (26%)] Rec: 4380.339, KLD: -0.020862, Loss: 4380.317871 Train Epoch: 19 [0576/1427 (39%)] Rec: 4495.019, KLD: 0.020862, Loss: 4495.039551 Train Epoch: 19 [0768/1427 (52%)] Rec: 4390.853, KLD: -0.008941, Loss: 4390.843750 Train Epoch: 19 [0960/1427 (65%)] Rec: 4328.123, KLD: 0.029802, Loss: 4328.152832 Train Epoch: 19 [1152/1427 (78%)] Rec: 4139.092, KLD: 0.047684, Loss: 4139.139648 Train Epoch: 19 [1344/1427 (91%)] Rec: 5219.827, KLD: 0.008941, Loss: 5219.835938 ====> Epoch: 19 Average loss: 72.61802 Training time for epoch: 28.28s ====================================================== Train Epoch: 20 [0000/1427 (0%)] Rec: 4379.958, KLD: 0.035763, Loss: 4379.993652 Train Epoch: 20 [0192/1427 (13%)] Rec: 4427.777, KLD: 0.035763, Loss: 4427.812988 Train Epoch: 20 [0384/1427 (26%)] Rec: 4352.665, KLD: 0.011921, Loss: 4352.676758 Train Epoch: 20 [0576/1427 (39%)] Rec: 4475.310, KLD: -0.005960, Loss: 4475.303711 Train Epoch: 20 [0768/1427 (52%)] Rec: 4389.233, KLD: 0.026822, Loss: 4389.259766 Train Epoch: 20 [0960/1427 (65%)] Rec: 4298.925, KLD: 0.008941, Loss: 4298.934082 Train Epoch: 20 [1152/1427 (78%)] Rec: 4125.768, KLD: 0.008941, Loss: 4125.776855 Train Epoch: 20 [1344/1427 (91%)] Rec: 5204.406, KLD: -0.017881, Loss: 5204.388184 ====> Epoch: 20 Average loss: 72.39108 Training time for epoch: 28.26s ====================================================== Train Epoch: 21 [0000/1427 (0%)] Rec: 4347.938, KLD: 0.065565, Loss: 4348.002930 Train Epoch: 21 [0192/1427 (13%)] Rec: 4382.589, KLD: 0.035763, Loss: 4382.625000 Train Epoch: 21 [0384/1427 (26%)] Rec: 4334.328, KLD: 0.029802, Loss: 4334.357910 Train Epoch: 21 [0576/1427 (39%)] Rec: 4475.529, KLD: 0.008941, Loss: 4475.537598 Train Epoch: 21 [0768/1427 (52%)] Rec: 4364.528, KLD: 0.005960, Loss: 4364.533691 Train Epoch: 21 [0960/1427 (65%)] Rec: 4269.968, KLD: 0.026822, Loss: 4269.994629 Train Epoch: 21 [1152/1427 (78%)] Rec: 4100.060, KLD: 0.047684, Loss: 4100.107422 Train Epoch: 21 [1344/1427 (91%)] Rec: 5168.366, KLD: -0.032783, Loss: 5168.333496 ====> Epoch: 21 Average loss: 72.00647 Training time for epoch: 28.16s ====================================================== Train Epoch: 22 [0000/1427 (0%)] Rec: 4342.573, KLD: 0.014901, Loss: 4342.588379 Train Epoch: 22 [0192/1427 (13%)] Rec: 4345.958, KLD: 0.053644, Loss: 4346.011719 Train Epoch: 22 [0384/1427 (26%)] Rec: 4313.161, KLD: -0.000000, Loss: 4313.161133 Train Epoch: 22 [0576/1427 (39%)] Rec: 4465.052, KLD: 0.008941, Loss: 4465.060547 Train Epoch: 22 [0768/1427 (52%)] Rec: 4352.885, KLD: 0.026822, Loss: 4352.911621 Train Epoch: 22 [0960/1427 (65%)] Rec: 4253.831, KLD: 0.026822, Loss: 4253.857910 Train Epoch: 22 [1152/1427 (78%)] Rec: 4081.558, KLD: 0.020862, Loss: 4081.578369 Train Epoch: 22 [1344/1427 (91%)] Rec: 5183.343, KLD: -0.002980, Loss: 5183.339844 ====> Epoch: 22 Average loss: 71.73201 Training time for epoch: 28.22s ====================================================== Train Epoch: 23 [0000/1427 (0%)] Rec: 4325.337, KLD: 0.038743, Loss: 4325.375977 Train Epoch: 23 [0192/1427 (13%)] Rec: 4335.301, KLD: 0.044703, Loss: 4335.345703 Train Epoch: 23 [0384/1427 (26%)] Rec: 4301.750, KLD: 0.023842, Loss: 4301.774414 Train Epoch: 23 [0576/1427 (39%)] Rec: 4460.478, KLD: 0.005960, Loss: 4460.483398 Train Epoch: 23 [0768/1427 (52%)] Rec: 4335.822, KLD: 0.023842, Loss: 4335.846191 Train Epoch: 23 [0960/1427 (65%)] Rec: 4226.401, KLD: 0.035763, Loss: 4226.436523 Train Epoch: 23 [1152/1427 (78%)] Rec: 4045.150, KLD: -0.020862, Loss: 4045.129150 Train Epoch: 23 [1344/1427 (91%)] Rec: 5149.463, KLD: -0.017881, Loss: 5149.445312 ====> Epoch: 23 Average loss: 71.40325 Training time for epoch: 28.21s ====================================================== Train Epoch: 24 [0000/1427 (0%)] Rec: 4301.287, KLD: 0.038743, Loss: 4301.325684 Train Epoch: 24 [0192/1427 (13%)] Rec: 4290.044, KLD: 0.014901, Loss: 4290.059570 Train Epoch: 24 [0384/1427 (26%)] Rec: 4234.220, KLD: -0.008941, Loss: 4234.210938 Train Epoch: 24 [0576/1427 (39%)] Rec: 4419.173, KLD: 0.029802, Loss: 4419.202637 Train Epoch: 24 [0768/1427 (52%)] Rec: 4315.383, KLD: 0.002980, Loss: 4315.386230 Train Epoch: 24 [0960/1427 (65%)] Rec: 4205.479, KLD: 0.020862, Loss: 4205.499512 Train Epoch: 24 [1152/1427 (78%)] Rec: 4012.175, KLD: 0.056624, Loss: 4012.231689 Train Epoch: 24 [1344/1427 (91%)] Rec: 5153.781, KLD: 0.008941, Loss: 5153.789551 ====> Epoch: 24 Average loss: 70.91280 Training time for epoch: 28.26s ====================================================== Train Epoch: 25 [0000/1427 (0%)] Rec: 4265.750, KLD: 0.041723, Loss: 4265.791504 Train Epoch: 25 [0192/1427 (13%)] Rec: 4255.803, KLD: 0.065565, Loss: 4255.868164 Train Epoch: 25 [0384/1427 (26%)] Rec: 4188.773, KLD: 0.032783, Loss: 4188.806152 Train Epoch: 25 [0576/1427 (39%)] Rec: 4394.392, KLD: 0.026822, Loss: 4394.418457 Train Epoch: 25 [0768/1427 (52%)] Rec: 4279.758, KLD: 0.017881, Loss: 4279.775879 Train Epoch: 25 [0960/1427 (65%)] Rec: 4144.127, KLD: -0.008941, Loss: 4144.118164 Train Epoch: 25 [1152/1427 (78%)] Rec: 3960.953, KLD: -0.017881, Loss: 3960.935303 Train Epoch: 25 [1344/1427 (91%)] Rec: 5111.450, KLD: 0.035763, Loss: 5111.485840 ====> Epoch: 25 Average loss: 70.16216 Training time for epoch: 29.10s ====================================================== Train Epoch: 26 [0000/1427 (0%)] Rec: 4193.157, KLD: 0.068545, Loss: 4193.225586 Train Epoch: 26 [0192/1427 (13%)] Rec: 4203.583, KLD: 0.053644, Loss: 4203.636719 Train Epoch: 26 [0384/1427 (26%)] Rec: 4080.267, KLD: -0.020862, Loss: 4080.246338 Train Epoch: 26 [0576/1427 (39%)] Rec: 4379.509, KLD: 0.017881, Loss: 4379.526855 Train Epoch: 26 [0768/1427 (52%)] Rec: 4241.943, KLD: 0.032783, Loss: 4241.976074 Train Epoch: 26 [0960/1427 (65%)] Rec: 4062.125, KLD: 0.002980, Loss: 4062.128174 Train Epoch: 26 [1152/1427 (78%)] Rec: 3891.709, KLD: 0.023842, Loss: 3891.732910 Train Epoch: 26 [1344/1427 (91%)] Rec: 5081.871, KLD: 0.011921, Loss: 5081.882812 ====> Epoch: 26 Average loss: 69.41517 Training time for epoch: 28.38s ====================================================== Train Epoch: 27 [0000/1427 (0%)] Rec: 4144.751, KLD: 0.023842, Loss: 4144.775391 Train Epoch: 27 [0192/1427 (13%)] Rec: 4125.910, KLD: 0.068545, Loss: 4125.978027 Train Epoch: 27 [0384/1427 (26%)] Rec: 4040.843, KLD: -0.005960, Loss: 4040.837158 Train Epoch: 27 [0576/1427 (39%)] Rec: 4345.512, KLD: 0.026822, Loss: 4345.539062 Train Epoch: 27 [0768/1427 (52%)] Rec: 4216.766, KLD: 0.053644, Loss: 4216.819336 Train Epoch: 27 [0960/1427 (65%)] Rec: 4031.610, KLD: 0.062585, Loss: 4031.672852 Train Epoch: 27 [1152/1427 (78%)] Rec: 3860.048, KLD: -0.029802, Loss: 3860.018066 Train Epoch: 27 [1344/1427 (91%)] Rec: 5012.964, KLD: 0.011921, Loss: 5012.976074 ====> Epoch: 27 Average loss: 68.73836 Training time for epoch: 28.28s ====================================================== Train Epoch: 28 [0000/1427 (0%)] Rec: 4124.688, KLD: 0.068545, Loss: 4124.756836 Train Epoch: 28 [0192/1427 (13%)] Rec: 4086.538, KLD: 0.080466, Loss: 4086.618164 Train Epoch: 28 [0384/1427 (26%)] Rec: 4011.520, KLD: 0.053644, Loss: 4011.573730 Train Epoch: 28 [0576/1427 (39%)] Rec: 4324.296, KLD: 0.023842, Loss: 4324.319824 Train Epoch: 28 [0768/1427 (52%)] Rec: 4194.101, KLD: 0.038743, Loss: 4194.139160 Train Epoch: 28 [0960/1427 (65%)] Rec: 3990.153, KLD: 0.023842, Loss: 3990.176758 Train Epoch: 28 [1152/1427 (78%)] Rec: 3860.566, KLD: 0.014901, Loss: 3860.580566 Train Epoch: 28 [1344/1427 (91%)] Rec: 4934.599, KLD: 0.029802, Loss: 4934.628418 ====> Epoch: 28 Average loss: 68.21121 Training time for epoch: 28.28s ====================================================== Train Epoch: 29 [0000/1427 (0%)] Rec: 4103.738, KLD: 0.038743, Loss: 4103.776855 Train Epoch: 29 [0192/1427 (13%)] Rec: 4042.153, KLD: 0.065565, Loss: 4042.218506 Train Epoch: 29 [0384/1427 (26%)] Rec: 3973.719, KLD: 0.005960, Loss: 3973.724365 Train Epoch: 29 [0576/1427 (39%)] Rec: 4308.748, KLD: -0.000000, Loss: 4308.747559 Train Epoch: 29 [0768/1427 (52%)] Rec: 4146.316, KLD: 0.014901, Loss: 4146.331543 Train Epoch: 29 [0960/1427 (65%)] Rec: 3985.406, KLD: 0.044703, Loss: 3985.450928 Train Epoch: 29 [1152/1427 (78%)] Rec: 3823.919, KLD: 0.008941, Loss: 3823.927734 Train Epoch: 29 [1344/1427 (91%)] Rec: 4943.005, KLD: 0.029802, Loss: 4943.034668 ====> Epoch: 29 Average loss: 67.84180 Training time for epoch: 28.23s ====================================================== Train Epoch: 30 [0000/1427 (0%)] Rec: 4072.671, KLD: 0.044703, Loss: 4072.715576 Train Epoch: 30 [0192/1427 (13%)] Rec: 4038.344, KLD: 0.056624, Loss: 4038.400879 Train Epoch: 30 [0384/1427 (26%)] Rec: 3939.877, KLD: 0.014901, Loss: 3939.892334 Train Epoch: 30 [0576/1427 (39%)] Rec: 4271.476, KLD: 0.068545, Loss: 4271.544434 Train Epoch: 30 [0768/1427 (52%)] Rec: 4166.829, KLD: 0.041723, Loss: 4166.870605 Train Epoch: 30 [0960/1427 (65%)] Rec: 3955.844, KLD: 0.032783, Loss: 3955.876953 Train Epoch: 30 [1152/1427 (78%)] Rec: 3803.041, KLD: 0.020862, Loss: 3803.062012 Train Epoch: 30 [1344/1427 (91%)] Rec: 4900.229, KLD: 0.026822, Loss: 4900.256348 ====> Epoch: 30 Average loss: 67.47607 Training time for epoch: 28.26s ====================================================== Train Epoch: 31 [0000/1427 (0%)] Rec: 4058.556, KLD: 0.011921, Loss: 4058.568359 Train Epoch: 31 [0192/1427 (13%)] Rec: 4011.302, KLD: 0.059605, Loss: 4011.362061 Train Epoch: 31 [0384/1427 (26%)] Rec: 3933.751, KLD: 0.014901, Loss: 3933.766113 Train Epoch: 31 [0576/1427 (39%)] Rec: 4274.380, KLD: -0.000000, Loss: 4274.379883 Train Epoch: 31 [0768/1427 (52%)] Rec: 4138.614, KLD: 0.047684, Loss: 4138.662109 Train Epoch: 31 [0960/1427 (65%)] Rec: 3955.036, KLD: 0.032783, Loss: 3955.068359 Train Epoch: 31 [1152/1427 (78%)] Rec: 3794.421, KLD: 0.011921, Loss: 3794.433105 Train Epoch: 31 [1344/1427 (91%)] Rec: 4900.260, KLD: 0.041723, Loss: 4900.301758 ====> Epoch: 31 Average loss: 67.18801 Training time for epoch: 28.26s ====================================================== Train Epoch: 32 [0000/1427 (0%)] Rec: 4065.248, KLD: 0.017881, Loss: 4065.265381 Train Epoch: 32 [0192/1427 (13%)] Rec: 3969.717, KLD: 0.044703, Loss: 3969.761475 Train Epoch: 32 [0384/1427 (26%)] Rec: 3932.668, KLD: 0.029802, Loss: 3932.697754 Train Epoch: 32 [0576/1427 (39%)] Rec: 4255.622, KLD: 0.017881, Loss: 4255.640137 Train Epoch: 32 [0768/1427 (52%)] Rec: 4138.777, KLD: 0.020862, Loss: 4138.798340 Train Epoch: 32 [0960/1427 (65%)] Rec: 3936.379, KLD: 0.059605, Loss: 3936.438477 Train Epoch: 32 [1152/1427 (78%)] Rec: 3782.352, KLD: 0.017881, Loss: 3782.369873 Train Epoch: 32 [1344/1427 (91%)] Rec: 4926.119, KLD: -0.000000, Loss: 4926.119141 ====> Epoch: 32 Average loss: 67.05698 Training time for epoch: 28.25s ====================================================== Train Epoch: 33 [0000/1427 (0%)] Rec: 4044.770, KLD: 0.020862, Loss: 4044.790771 Train Epoch: 33 [0192/1427 (13%)] Rec: 3975.695, KLD: 0.047684, Loss: 3975.742676 Train Epoch: 33 [0384/1427 (26%)] Rec: 3908.936, KLD: -0.002980, Loss: 3908.932617 Train Epoch: 33 [0576/1427 (39%)] Rec: 4262.639, KLD: 0.044703, Loss: 4262.683594 Train Epoch: 33 [0768/1427 (52%)] Rec: 4132.521, KLD: 0.029802, Loss: 4132.550293 Train Epoch: 33 [0960/1427 (65%)] Rec: 3924.888, KLD: 0.032783, Loss: 3924.920898 Train Epoch: 33 [1152/1427 (78%)] Rec: 3786.523, KLD: 0.008941, Loss: 3786.531982 Train Epoch: 33 [1344/1427 (91%)] Rec: 4880.928, KLD: 0.032783, Loss: 4880.960449 ====> Epoch: 33 Average loss: 66.92349 Training time for epoch: 28.26s ====================================================== Train Epoch: 34 [0000/1427 (0%)] Rec: 4025.491, KLD: 0.023842, Loss: 4025.514648 Train Epoch: 34 [0192/1427 (13%)] Rec: 3953.001, KLD: 0.047684, Loss: 3953.048584 Train Epoch: 34 [0384/1427 (26%)] Rec: 3895.434, KLD: 0.002980, Loss: 3895.436768 Train Epoch: 34 [0576/1427 (39%)] Rec: 4242.276, KLD: 0.047684, Loss: 4242.324219 Train Epoch: 34 [0768/1427 (52%)] Rec: 4099.573, KLD: 0.044703, Loss: 4099.618164 Train Epoch: 34 [0960/1427 (65%)] Rec: 3919.910, KLD: -0.000000, Loss: 3919.909912 Train Epoch: 34 [1152/1427 (78%)] Rec: 3758.585, KLD: -0.011921, Loss: 3758.573486 Train Epoch: 34 [1344/1427 (91%)] Rec: 4905.546, KLD: 0.017881, Loss: 4905.563965 ====> Epoch: 34 Average loss: 66.74415 Training time for epoch: 28.14s ====================================================== Train Epoch: 35 [0000/1427 (0%)] Rec: 4025.320, KLD: 0.044703, Loss: 4025.364990 Train Epoch: 35 [0192/1427 (13%)] Rec: 3943.988, KLD: 0.041723, Loss: 3944.029297 Train Epoch: 35 [0384/1427 (26%)] Rec: 3882.209, KLD: 0.008941, Loss: 3882.218506 Train Epoch: 35 [0576/1427 (39%)] Rec: 4240.766, KLD: 0.032783, Loss: 4240.798340 Train Epoch: 35 [0768/1427 (52%)] Rec: 4082.919, KLD: 0.032783, Loss: 4082.951416 Train Epoch: 35 [0960/1427 (65%)] Rec: 3891.542, KLD: 0.044703, Loss: 3891.587158 Train Epoch: 35 [1152/1427 (78%)] Rec: 3749.430, KLD: 0.014901, Loss: 3749.444580 Train Epoch: 35 [1344/1427 (91%)] Rec: 4861.264, KLD: 0.020862, Loss: 4861.285156 ====> Epoch: 35 Average loss: 66.60350 Training time for epoch: 28.22s ====================================================== Train Epoch: 36 [0000/1427 (0%)] Rec: 4015.821, KLD: 0.047684, Loss: 4015.868408 Train Epoch: 36 [0192/1427 (13%)] Rec: 3939.987, KLD: 0.074506, Loss: 3940.061035 Train Epoch: 36 [0384/1427 (26%)] Rec: 3860.790, KLD: 0.017881, Loss: 3860.807617 Train Epoch: 36 [0576/1427 (39%)] Rec: 4221.463, KLD: 0.038743, Loss: 4221.501465 Train Epoch: 36 [0768/1427 (52%)] Rec: 4093.616, KLD: 0.023842, Loss: 4093.639893 Train Epoch: 36 [0960/1427 (65%)] Rec: 3889.181, KLD: 0.071526, Loss: 3889.252686 Train Epoch: 36 [1152/1427 (78%)] Rec: 3730.717, KLD: 0.005960, Loss: 3730.723145 Train Epoch: 36 [1344/1427 (91%)] Rec: 4845.204, KLD: 0.002980, Loss: 4845.207031 ====> Epoch: 36 Average loss: 66.40742 Training time for epoch: 28.21s ====================================================== Train Epoch: 37 [0000/1427 (0%)] Rec: 4014.521, KLD: 0.032783, Loss: 4014.554199 Train Epoch: 37 [0192/1427 (13%)] Rec: 3921.180, KLD: 0.056624, Loss: 3921.237061 Train Epoch: 37 [0384/1427 (26%)] Rec: 3876.266, KLD: 0.002980, Loss: 3876.269043 Train Epoch: 37 [0576/1427 (39%)] Rec: 4214.917, KLD: 0.023842, Loss: 4214.940430 Train Epoch: 37 [0768/1427 (52%)] Rec: 4097.261, KLD: -0.020862, Loss: 4097.239746 Train Epoch: 37 [0960/1427 (65%)] Rec: 3887.000, KLD: 0.032783, Loss: 3887.032715 Train Epoch: 37 [1152/1427 (78%)] Rec: 3746.952, KLD: -0.017881, Loss: 3746.933838 Train Epoch: 37 [1344/1427 (91%)] Rec: 4873.521, KLD: 0.041723, Loss: 4873.562988 ====> Epoch: 37 Average loss: 66.37811 Training time for epoch: 28.29s ====================================================== Train Epoch: 38 [0000/1427 (0%)] Rec: 4014.810, KLD: 0.032783, Loss: 4014.842773 Train Epoch: 38 [0192/1427 (13%)] Rec: 3918.608, KLD: 0.062585, Loss: 3918.670410 Train Epoch: 38 [0384/1427 (26%)] Rec: 3884.989, KLD: 0.023842, Loss: 3885.013184 Train Epoch: 38 [0576/1427 (39%)] Rec: 4199.905, KLD: 0.032783, Loss: 4199.937988 Train Epoch: 38 [0768/1427 (52%)] Rec: 4103.084, KLD: 0.017881, Loss: 4103.102539 Train Epoch: 38 [0960/1427 (65%)] Rec: 3887.792, KLD: 0.038743, Loss: 3887.831299 Train Epoch: 38 [1152/1427 (78%)] Rec: 3726.745, KLD: 0.002980, Loss: 3726.748291 Train Epoch: 38 [1344/1427 (91%)] Rec: 4844.274, KLD: 0.017881, Loss: 4844.292480 ====> Epoch: 38 Average loss: 66.30684 Training time for epoch: 28.25s ====================================================== Train Epoch: 39 [0000/1427 (0%)] Rec: 4006.320, KLD: 0.059605, Loss: 4006.379150 Train Epoch: 39 [0192/1427 (13%)] Rec: 3918.456, KLD: 0.062585, Loss: 3918.518555 Train Epoch: 39 [0384/1427 (26%)] Rec: 3861.313, KLD: 0.011921, Loss: 3861.325439 Train Epoch: 39 [0576/1427 (39%)] Rec: 4186.067, KLD: 0.035763, Loss: 4186.103027 Train Epoch: 39 [0768/1427 (52%)] Rec: 4092.453, KLD: 0.005960, Loss: 4092.458740 Train Epoch: 39 [0960/1427 (65%)] Rec: 3881.064, KLD: 0.008941, Loss: 3881.073242 Train Epoch: 39 [1152/1427 (78%)] Rec: 3736.519, KLD: -0.002980, Loss: 3736.516357 Train Epoch: 39 [1344/1427 (91%)] Rec: 4832.663, KLD: 0.017881, Loss: 4832.681152 ====> Epoch: 39 Average loss: 66.15196 Training time for epoch: 28.32s ====================================================== Train Epoch: 40 [0000/1427 (0%)] Rec: 4009.232, KLD: 0.038743, Loss: 4009.270752 Train Epoch: 40 [0192/1427 (13%)] Rec: 3892.476, KLD: 0.050664, Loss: 3892.527100 Train Epoch: 40 [0384/1427 (26%)] Rec: 3852.266, KLD: -0.014901, Loss: 3852.250977 Train Epoch: 40 [0576/1427 (39%)] Rec: 4202.350, KLD: 0.047684, Loss: 4202.397949 Train Epoch: 40 [0768/1427 (52%)] Rec: 4075.646, KLD: 0.023842, Loss: 4075.669678 Train Epoch: 40 [0960/1427 (65%)] Rec: 3871.795, KLD: 0.020862, Loss: 3871.816162 Train Epoch: 40 [1152/1427 (78%)] Rec: 3713.910, KLD: 0.002980, Loss: 3713.912598 Train Epoch: 40 [1344/1427 (91%)] Rec: 4804.302, KLD: -0.005960, Loss: 4804.295898 ====> Epoch: 40 Average loss: 66.01916 Training time for epoch: 28.11s ====================================================== Train Epoch: 41 [0000/1427 (0%)] Rec: 4018.196, KLD: 0.011921, Loss: 4018.208252 Train Epoch: 41 [0192/1427 (13%)] Rec: 3894.513, KLD: 0.038743, Loss: 3894.552246 Train Epoch: 41 [0384/1427 (26%)] Rec: 3863.633, KLD: -0.014901, Loss: 3863.617920 Train Epoch: 41 [0576/1427 (39%)] Rec: 4184.485, KLD: 0.062585, Loss: 4184.547363 Train Epoch: 41 [0768/1427 (52%)] Rec: 4093.609, KLD: 0.044703, Loss: 4093.653320 Train Epoch: 41 [0960/1427 (65%)] Rec: 3890.058, KLD: 0.005960, Loss: 3890.063721 Train Epoch: 41 [1152/1427 (78%)] Rec: 3712.676, KLD: 0.029802, Loss: 3712.705322 Train Epoch: 41 [1344/1427 (91%)] Rec: 4864.593, KLD: 0.008941, Loss: 4864.602051 ====> Epoch: 41 Average loss: 66.06073 Training time for epoch: 28.01s ====================================================== Train Epoch: 42 [0000/1427 (0%)] Rec: 4003.633, KLD: 0.041723, Loss: 4003.675049 Train Epoch: 42 [0192/1427 (13%)] Rec: 3906.553, KLD: 0.044703, Loss: 3906.597900 Train Epoch: 42 [0384/1427 (26%)] Rec: 3852.491, KLD: 0.008941, Loss: 3852.500244 Train Epoch: 42 [0576/1427 (39%)] Rec: 4157.104, KLD: 0.032783, Loss: 4157.136230 Train Epoch: 42 [0768/1427 (52%)] Rec: 4086.681, KLD: 0.035763, Loss: 4086.717041 Train Epoch: 42 [0960/1427 (65%)] Rec: 3883.251, KLD: 0.002980, Loss: 3883.253662 Train Epoch: 42 [1152/1427 (78%)] Rec: 3706.988, KLD: 0.011921, Loss: 3706.999756 Train Epoch: 42 [1344/1427 (91%)] Rec: 4822.143, KLD: 0.014901, Loss: 4822.157715 ====> Epoch: 42 Average loss: 65.97366 Training time for epoch: 28.11s ====================================================== Train Epoch: 43 [0000/1427 (0%)] Rec: 4007.696, KLD: 0.038743, Loss: 4007.735107 Train Epoch: 43 [0192/1427 (13%)] Rec: 3904.126, KLD: 0.065565, Loss: 3904.192139 Train Epoch: 43 [0384/1427 (26%)] Rec: 3855.742, KLD: 0.026822, Loss: 3855.768555 Train Epoch: 43 [0576/1427 (39%)] Rec: 4166.613, KLD: 0.044703, Loss: 4166.658203 Train Epoch: 43 [0768/1427 (52%)] Rec: 4086.439, KLD: 0.053644, Loss: 4086.492676 Train Epoch: 43 [0960/1427 (65%)] Rec: 3876.682, KLD: 0.020862, Loss: 3876.702637 Train Epoch: 43 [1152/1427 (78%)] Rec: 3717.759, KLD: 0.032783, Loss: 3717.791992 Train Epoch: 43 [1344/1427 (91%)] Rec: 4784.347, KLD: 0.020862, Loss: 4784.367676 ====> Epoch: 43 Average loss: 65.83491 Training time for epoch: 28.65s ====================================================== Train Epoch: 44 [0000/1427 (0%)] Rec: 4002.085, KLD: 0.026822, Loss: 4002.112305 Train Epoch: 44 [0192/1427 (13%)] Rec: 3886.490, KLD: 0.056624, Loss: 3886.547119 Train Epoch: 44 [0384/1427 (26%)] Rec: 3842.454, KLD: 0.008941, Loss: 3842.463379 Train Epoch: 44 [0576/1427 (39%)] Rec: 4155.444, KLD: 0.005960, Loss: 4155.450195 Train Epoch: 44 [0768/1427 (52%)] Rec: 4092.882, KLD: 0.017881, Loss: 4092.899902 Train Epoch: 44 [0960/1427 (65%)] Rec: 3893.075, KLD: 0.005960, Loss: 3893.081299 Train Epoch: 44 [1152/1427 (78%)] Rec: 3709.935, KLD: 0.023842, Loss: 3709.959229 Train Epoch: 44 [1344/1427 (91%)] Rec: 4806.825, KLD: 0.020862, Loss: 4806.846191 ====> Epoch: 44 Average loss: 65.78828 Training time for epoch: 29.21s ====================================================== Train Epoch: 45 [0000/1427 (0%)] Rec: 3978.354, KLD: 0.035763, Loss: 3978.389893 Train Epoch: 45 [0192/1427 (13%)] Rec: 3908.773, KLD: 0.059605, Loss: 3908.832275 Train Epoch: 45 [0384/1427 (26%)] Rec: 3837.212, KLD: 0.029802, Loss: 3837.241699 Train Epoch: 45 [0576/1427 (39%)] Rec: 4140.173, KLD: 0.026822, Loss: 4140.199707 Train Epoch: 45 [0768/1427 (52%)] Rec: 4088.109, KLD: 0.059605, Loss: 4088.168213 Train Epoch: 45 [0960/1427 (65%)] Rec: 3894.896, KLD: 0.017881, Loss: 3894.914307 Train Epoch: 45 [1152/1427 (78%)] Rec: 3702.910, KLD: 0.020862, Loss: 3702.930908 Train Epoch: 45 [1344/1427 (91%)] Rec: 4769.430, KLD: 0.014901, Loss: 4769.444824 ====> Epoch: 45 Average loss: 65.76067 Training time for epoch: 28.72s ====================================================== Train Epoch: 46 [0000/1427 (0%)] Rec: 3994.316, KLD: 0.026822, Loss: 3994.342773 Train Epoch: 46 [0192/1427 (13%)] Rec: 3914.934, KLD: 0.029802, Loss: 3914.963379 Train Epoch: 46 [0384/1427 (26%)] Rec: 3833.982, KLD: 0.002980, Loss: 3833.984863 Train Epoch: 46 [0576/1427 (39%)] Rec: 4144.903, KLD: 0.032783, Loss: 4144.935547 Train Epoch: 46 [0768/1427 (52%)] Rec: 4070.742, KLD: 0.017881, Loss: 4070.760254 Train Epoch: 46 [0960/1427 (65%)] Rec: 3901.204, KLD: 0.044703, Loss: 3901.248779 Train Epoch: 46 [1152/1427 (78%)] Rec: 3706.867, KLD: 0.005960, Loss: 3706.872559 Train Epoch: 46 [1344/1427 (91%)] Rec: 4775.105, KLD: 0.014901, Loss: 4775.120117 ====> Epoch: 46 Average loss: 65.72444 Training time for epoch: 28.68s ====================================================== Train Epoch: 47 [0000/1427 (0%)] Rec: 3977.931, KLD: 0.047684, Loss: 3977.978271 Train Epoch: 47 [0192/1427 (13%)] Rec: 3903.150, KLD: 0.044703, Loss: 3903.195068 Train Epoch: 47 [0384/1427 (26%)] Rec: 3843.193, KLD: 0.005960, Loss: 3843.199219 Train Epoch: 47 [0576/1427 (39%)] Rec: 4141.552, KLD: 0.041723, Loss: 4141.593262 Train Epoch: 47 [0768/1427 (52%)] Rec: 4062.405, KLD: 0.023842, Loss: 4062.428711 Train Epoch: 47 [0960/1427 (65%)] Rec: 3907.453, KLD: 0.014901, Loss: 3907.468018 Train Epoch: 47 [1152/1427 (78%)] Rec: 3711.925, KLD: 0.032783, Loss: 3711.958008 Train Epoch: 47 [1344/1427 (91%)] Rec: 4799.992, KLD: 0.014901, Loss: 4800.007324 ====> Epoch: 47 Average loss: 65.70659 Training time for epoch: 27.62s ====================================================== Train Epoch: 48 [0000/1427 (0%)] Rec: 3957.629, KLD: 0.035763, Loss: 3957.665039 Train Epoch: 48 [0192/1427 (13%)] Rec: 3895.159, KLD: 0.038743, Loss: 3895.197998 Train Epoch: 48 [0384/1427 (26%)] Rec: 3851.471, KLD: 0.023842, Loss: 3851.494629 Train Epoch: 48 [0576/1427 (39%)] Rec: 4140.117, KLD: 0.068545, Loss: 4140.185547 Train Epoch: 48 [0768/1427 (52%)] Rec: 4058.464, KLD: 0.038743, Loss: 4058.502441 Train Epoch: 48 [0960/1427 (65%)] Rec: 3917.583, KLD: 0.041723, Loss: 3917.624268 Train Epoch: 48 [1152/1427 (78%)] Rec: 3697.169, KLD: 0.002980, Loss: 3697.171875 Train Epoch: 48 [1344/1427 (91%)] Rec: 4787.321, KLD: -0.023842, Loss: 4787.297363 ====> Epoch: 48 Average loss: 65.73428 Training time for epoch: 27.80s ====================================================== Train Epoch: 49 [0000/1427 (0%)] Rec: 3963.367, KLD: 0.062585, Loss: 3963.429443 Train Epoch: 49 [0192/1427 (13%)] Rec: 3923.073, KLD: 0.059605, Loss: 3923.132812 Train Epoch: 49 [0384/1427 (26%)] Rec: 3840.030, KLD: 0.008941, Loss: 3840.039307 Train Epoch: 49 [0576/1427 (39%)] Rec: 4166.864, KLD: 0.053644, Loss: 4166.917969 Train Epoch: 49 [0768/1427 (52%)] Rec: 4050.441, KLD: 0.005960, Loss: 4050.446777 Train Epoch: 49 [0960/1427 (65%)] Rec: 3923.044, KLD: 0.014901, Loss: 3923.058838 Train Epoch: 49 [1152/1427 (78%)] Rec: 3710.494, KLD: 0.002980, Loss: 3710.497070 Train Epoch: 49 [1344/1427 (91%)] Rec: 4769.342, KLD: 0.002980, Loss: 4769.344727 ====> Epoch: 49 Average loss: 65.77142 Training time for epoch: 27.71s ======================================================
In [11]:
#plot loss
SHOW_FROM = 0
plt.plot(loss_history[SHOW_FROM:], label="loss")
plt.plot(rec_loss_history[SHOW_FROM:], label="rec")
plt.plot(kl_loss_history[SHOW_FROM:], label="kl")
plt.xlabel("iters")
plt.legend(loc='upper right')
Out[11]:
<matplotlib.legend.Legend at 0x2822d26c320>
In [12]:
#Save checkpoint
if EPOCHS != 0:
ut.save_checkpoint(results_dir, epoch, model, optimizer, loss_history, rec_loss_history, kl_loss_history)
print('Save checkpoint at epoch ',epoch)
Save checkpoint at epoch 49
In [13]:
#Check reconstruction
inbatch_idx = 0
with torch.no_grad():
x = next(iter(train_loader))
x = x.to(device)
recon_x, _, _ = model(x)
plot_3d_mesh(x[inbatch_idx])
plot_3d_mesh(recon_x[inbatch_idx])
In [14]:
#CHECK GENERATION RESULTS
print('Generating samples')
N_samples = 3
with torch.no_grad():
z_samples = torch.randn(N_samples, Z_DIMS).to(device)
samples = model.decode(z_samples)
for x in samples:
plot_3d_mesh(x)
# plot_3d_mesh(ut.do_threshold(x, eps=0.3))
# plot_3d_mesh(np.round(x))
Generating samples
In [15]:
sample = samples[0][0]
thresholded = ut.do_threshold(sample, eps=0.3)
ut.plot_voxels_views(thresholded)
